/*
 * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved.
 * Description: transport mem
 * Author: yanzhenghang
 * Create: 2024-09-06
 */

#ifndef TRANSPORT_ROCE_MEM_H
#define TRANSPORT_ROCE_MEM_H

#include "transport_mem.h"

namespace hccl {
constexpr u32 REMOTE_RDMA_SIGNAL_SIZE = 1;
constexpr u64 MAX_RDMA_WQE_SIZE = 2ULL * 1024 * 1024 * 1024; // RDMA最大WQE限制, 2G限制是RDMA导致

class TransportRoceMem : public TransportMem {
public:
    explicit TransportRoceMem(const std::unique_ptr<NotifyPool> &notifyPool, const HcclNetDevCtx &netDevCtx,
        const HcclDispatcher &dispatcher, u32 localRankId, u32 remoteRankId);
    ~TransportRoceMem() override;
    HcclResult SetSocket(const std::shared_ptr<HcclSocket> &socket) override;
    HcclResult Connect(s32 timeoutSec) override;
    HcclResult Write(const RmaBufferSlice &remoteRmaBufferSlice, const RmaBufferSlice &localRmaBufferSlice, const rtStream_t &stream) override;
    HcclResult Read(const RmaBufferSlice &localRmaBufferSlice, const RmaBufferSlice &remoteRmaBufferSlice, const rtStream_t &stream) override;
    HcclResult AddOpFence(const rtStream_t &stream) override;

private:
    enum class RdmaOp {
        OP_WRITE = 0,
        OP_SEND = 2,
        OP_READ = 4
    };
    HcclResult CreateCqAndQp();
    HcclResult WaitQPLinkComplete(s32 timeoutSec);
    HcclResult DoorBellSend(const s32 qpMode, const send_wr_v2 &sendWrV2, const send_wr_rsp &opRsp, void *stream);
    HcclResult DestroyCqAndQp();
    HcclResult CreatSignalMesg();
    HcclResult GetNotifySize();
    HcclResult RecoverNotifyMsg(MemMsg *remoteRdmaSignal, u64 signalNum);

    HcclResult GetQpStatus();
    HcclResult QpConnect();
    HcclResult CreateRdmaSignal(
        std::shared_ptr<LocalIpcNotify> &localNotify, MemMsg &rdmaSignalInfo, MemType notifyType);
    HcclResult GetRdmaHandle();
    HcclResult ExchangeNotifyValueBuffer();
    HcclResult RdmaDbSend(u32 dbindex, u64 dbinfo, const struct send_wr &sendWr, rtStream_t stream);
    HcclResult CreateNotifyValueBuffer();
    HcclResult TransportRdmaWithType(const RmaBufferSlice &localRmaBufferSlice, const RmaBufferSlice &remoteRmaBufferSlice, const rtStream_t &stream, const RdmaOp &rdmaOp);
    HcclResult TransportIpc(const RmaBufferSlice &dstRmaBufferSlice, const RmaBufferSlice &srcRmaBufferSlice, const rtStream_t &stream);

    std::array<MemMsg, static_cast<u32>(MemType::MEM_TYPE_RESERVED)> notifyMemMsg_;
    QpInfo dataQpInfo_{};
    s32 access_{RA_ACCESS_LOCAL_WRITE | RA_ACCESS_REMOTE_WRITE | RA_ACCESS_REMOTE_READ};
    void *nicRdmaHandle_{nullptr};
    std::shared_ptr<LocalIpcNotify> remoteIsendDoneSignal_{nullptr};
    MemMsg rdmaSignal_[REMOTE_RDMA_SIGNAL_SIZE];
    MrHandle rdmaSignalMrHandle_{nullptr};
    MrHandle notifyValueMemMrHandle_{nullptr};
    u32 notifySize_{0};
    s32 deviceLogicId_{-1};
    s32 devicePhyId_{-1};
    s32 index_{-1};
    static std::atomic<uint64_t> sendWrHandle;
    std::chrono::seconds maxTimeOut_{0};
    const u64 notifyValueSize_{LARGE_PAGE_MEMORY_MIN_SIZE};
    DeviceMem notifyMem_{};
};
}  // namespace hccl
#endif